'''
爬取豆瓣电影内容
如果获取javascript渲染后的内容：使用selenium的page_source
# 使用py获取属性名=属性值的标签
    movie_type = info('span[property="v:genre"]').text()
'''
import requests  # requests库没办法解决script渲染的结果
import time
from selenium import webdriver
from lxml import etree
from pyquery import PyQuery as pq
import re
import pymysql


# 获取javascript渲染后的内容弄
def get_response(url):
    option = webdriver.ChromeOptions()
    option.add_argument('--headless')
    browser = webdriver.Chrome(options=option)
    browser.get(url)
    time.sleep(2)
    return browser.page_source


# 获取正常的静态网页
def get_permovie_response(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
        'Referer': 'https://www.douban.com/'
    }
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except:
        print('请求出错')
        return None


# 解析网页获取每一个信息
def parse_page(html):
    text = etree.HTML(html)
    slide_wrapper = text.xpath('//div[@class="slide-wrapper"]')[0]
    slides = slide_wrapper.xpath('./div[@class="slide-page"]')
    # [start,end]包前不包后，所以取到的结果为start到end-1
    movie_list = []
    for slide in slides[1:-1]:
        for a in slide:
            # print(a.xpath('.//text()'))
            # xpath返回的是一个列表需要进行提取
            href = a.xpath('.//@href')[0]
            img = a.xpath('./div/img/@src')[0]
            title = ''.join(a.xpath('./p/text()')).strip()
            score = str(a.xpath('./p/strong/text()')[0])
            movie_html = get_permovie_response(href)
            detail_info = parser_movie(movie_html)

            movie = {
                'title': title,
                'href': href,
                'img': img,
                'score': score,
                'detail_info': detail_info  # 字典类型
            }
            print(movie)
            # 一个一个存，还是全部获取完成后再存，一个传递的是dict，一个传递的是list
            save_to_mysql(movie)
            movie_list.append(movie)


# 解析每一个电影的具体信息
def parser_movie(html):
    doc = pq(html)
    info = doc('#info')
    introduct = doc('#link-report')
    director = info('span:first-child .attrs a').text()
    editors = info('span .attrs').eq(1).find('a').items()
    editor_list = []
    for editor in editors:
        editor_list.append(editor.text())
    editor_list = ','.join(editor_list)
    actors = info('span .attrs').eq(2).find('a').items()
    actor_list = []
    for actor in actors:
        actor_list.append(actor.text())
    actor_list = ','.join(actor_list)
    # 使用py获取属性名=属性值的标签
    movie_type = info('span[property="v:genre"]').text()
    address = re.search(r'<span class="pl">制片国家/地区:</span>(.*?)<br/>', str(info)).group(1).strip()
    language = re.search(r'<span class="pl">语言:</span>(.*?)<br/>', str(info)).group(1).strip()
    time = info('span[property="v:initialReleaseDate"]').text()
    runtime = info('span[property="v:runtime"]').text()
    summary = introduct('span[property="v:summary"]').text()
    main_info = {
        'director': director,
        'editor_list': editor_list,
        'actor_list': actor_list,
        'movie_type': movie_type,
        'address': address,
        'language': language,
        'time': time,
        'runtime': runtime,
        'summary': summary
    }
    return main_info


def save_to_mysql(movie):
    db = pymysql.connect('localhost', 'root', 'root', 'test')
    cursor = db.cursor()
    sql = 'insert into douban_movie(title, href, img, score, director, editor_list, actor_list, movie_type, address, ' \
          'language, time, runtime, summary) VALUES("{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}")'\
        .format(movie['title'], movie['href'], movie['img'], movie['score'], movie['detail_info']['director'],
                movie['detail_info']['editor_list'], movie['detail_info']['actor_list'], movie['detail_info']['movie_type'],
                movie['detail_info']['address'], movie['detail_info']['language'], movie['detail_info']['time'],
                movie['detail_info']['runtime'], movie['detail_info']['summary'])
    try:
        cursor.execute(sql)
        db.commit()
    except:
        print('mysql插入出错')
        db.rollback()
    db.close()



def main():
    url = 'https://movie.douban.com/'
    html = get_response(url)
    parse_page(html)

if __name__ == '__main__':
    main()